#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""Dourbes data format
Copyright (c) 2011-2015 Christian T. Steigies <steigies@physik.uni-kiel.de>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.

This module offers functions to read NM data in CSV format as used in South Africa
and return data in the format as used by NMDB.
The CSV files are (symlinked) in ~/data/south_africa/STATION/[1h|ori|rev]
"""

__author__ = "Christian T. Steigies <steigies@physik.uni-kiel.de>"
__license__ = "GPL License"

import os
from datetime import datetime
#import urllib2
import numpy as np

from nmdb.station.station_data import check_data, check_pressure
#from nmdb.tools.configtool import parse_config
from nmdb.tools.datetool import valid_date, year2
from nmdb.tools.pressure import mbar

dourbes_station = ["drbs"]
dourbes_longnames = {"drbs": "Dourbes"}


## --------------------------------------------------------------------------
def read_rt(options, station, data_type, year, month, day):
    """read realtime data

    options     verbose, dryrun, ...
    station     station short name, must be valid station
    data_type   "1m", "1h", "env", "meta"
    year        integer (1950-9999), may not be None
    month       integer (1-12), if None, read month 1-12
    day         integer (1-31), if None, read day 1-31

    return list of values in nmdb format
    """

    if station in dourbes_station:
        pass
    else:
        raise ValueError  # wrong station

    value = []
    if valid_date(year, month, day):  # real-time data is after 2013
        value = read_daily(options, station, data_type, year, month, day)

    return value

## --------------------------------------------------------------------------
def read_rev(options, station, data_type, year, month, day):
    """read revised data

    options     verbose, dryrun, ...
    station     station short name, must be valid station
    data_type   "1m", "1h", "env", "meta"
    year        integer (1950-9999), may not be None
    month       integer (1-12), if None, read month 1-12
    day         integer (1-31), if None, read day 1-31

    return list of values in nmdb format
    """

    if station in dourbes_station:
        pass
    else:
        raise ValueError  # wrong station

    value = []
    if options.verbose > 1:
        print "drbs read_rev", year, month, day

    if valid_date(year, month, day):
        if year <= 2012:
            #value = read_corrected(options, station, data_type, year, month, day)
            value = read_yearly(options, station, data_type, year, month, day)
        else:
            value = read_daily(options, station, data_type, year, month, day)

    return value

## --------------------------------------------------------------------------
def split_corrected(options, station, data_type):
    """split corrected_data into yearly files, use with read_yearly"""

    data = []
    data_path = '~/data/dourbes'

    data_path = os.path.expanduser(data_path)
    if data_type in ["1m", "ori", "rev"]:
        data_type = "1m"
#        filename = '~/Data/soft-derived/cosmic-rays/corrected-intensity/DOUR@NMDB/nm64-v1/1m/corrected_data_till_2012_10_09.dat'
        filename = '~/data/dourbes/1m/corrected_data_till_2012_10_09.dat'
        filename = os.path.expanduser(filename)
    else:
        print "wrong data type"
        return data

    try:
        myfile = open(filename, 'rb')
    except IOError:
        print "file not found", filename
        return data

    prev_year = 0
    line = True
    while line:
        line = myfile.readline()
        row = line.split()
        if (len(line) < 14) or (len(line) > 100):
            continue  # skip, line too short or too long for data
        date=row[0].split("/")
        year = int(date[2])
        if year != prev_year:
            filename = "%s/corrected_%i.dat" % (data_path, year)
            print "new file", filename
            prev_year = year
            data.append(year)
            try:
                outfile = open(filename, 'w')
            except IOError:
                print "could not open file for writing:", filename
        try:
            outfile.write(line)
        except IOError:
            print "file not open error"

    return data

## --------------------------------------------------------------------------
def read_1h(options, station, year, month, day):
    """calculate 1h data from 1m/15m

    options     verbose, dryrun, ...
    station     station short name, must be valid station
    year        integer (1950-9999), may not be None
    month       integer (1-12), if None, read month 1-12
    day         integer (1-31), if None, read day 1-31

    return list of values in nmdb format
    """

    if station in dourbes_station:
        pass
    else:
        raise ValueError  # wrong station

    value = []
    if valid_date(year, month, day):  # real-time data is after 2013
        value = read_rev(options, station, "1m", year, month, day)

    data = []
    for hour in range(0,24):
        data_u = []
        data_c = []
        data_e = []
        data_p = []
        for line in value:
            (timestamp, interval, uncorr, ceff, corr, press) = line
            if timestamp >= datetime(year, month, day, hour, 0) and timestamp <= datetime(year, month, day, hour, 59, 59):
                if uncorr is not None:
                    data_u.append(uncorr)
                if ceff is not None:
                    data_e.append(ceff)
                if corr is not None:
                    data_c.append(corr)
                if press is not None:
                    data_p.append(press)
        data.append([datetime(year, month, day, hour, 0),
                     3600,
                     np.average(data_u),
                     np.average(data_e),
                     np.average(data_c),
                     np.average(data_p)])
    return data

## --------------------------------------------------------------------------
def read_yearly(options, station, data_type, year=None, month=None, day=None):
    """read one year of data from split files with corrected data

    DD/MM/YYYY hh:mm[:ss] p_mbar ucorr corr,delta
    ~/Data/soft-derived/cosmic-rays/corrected-intensity/DOUR@NMDB/nm64-v1/1m/corrected_data_till_2012_10_09.dat 
    1/10/1965 00:00 983.485 6805.18 6651.19 -0.0388046
    1/10/1965 00:15 983.631 6815.44 6668.43 -0.0363137
...
31/12/2008 23:30        1002.25 6240.72 7006.72 0.0125734       
31/12/2008 23:45        1002.15 6199.34 6954.91 0.00508661      
2/01/2009       00:00:00        1000.12 6120    6763.74 -0.0225394
2/01/2009       00:01:00        1000.13 6213    6867.03 -0.00761256
...
9/10/2012       23:58:00        989.55  6177    6313.91 -0.0875475                      
9/10/2012       23:59:00        989.53  6290    6428.46 -0.0709927   
    """
    
    data = []    
    data_path = '~/data/dourbes'

    data_path = os.path.expanduser(data_path)
    if data_type in ["1m", "ori", "rev"]:
        data_type = "1m"
        filename = "%s/corrected_%i.dat" % (data_path, year)
        if year <= 2008:
            interval = 60*15
        else:
            interval = 60
    elif data_type in["1h"]:
        data = read_1h(options, station, year, month, day)
        return data
    else:
        print "unknown data:", data_type
        return data

    try:
        myfile = open(filename, 'rb')
    except IOError:
        print "file not found", filename
        return data        

    line = True
    while line:
        line = myfile.readline()
        if (len(line) < 14) or (len(line) > 100):
            continue  # skip, line too short or too long for data
        row = line.split()

        date=row[0].split("/")
        if year and year != int(date[2]):
            continue  # skip data if year does not match
        if month and month != int(date[1]):
            continue  # skip data if month does not match
        if day and day != int(date[0]):
            continue  # skip data if day does not match

        time = row[1].split(":")
        hour = int(time[0])
        minute = int(time[1])

        timestamp = datetime(year, month, day, hour, minute)

        try:
            press = float(row[2])
        except ValueError:
            press = 0
        try:
            ucorr = float(row[3])
        except ValueError:
            ucorr = 0
        try:
            corr = float(row[4])
        except ValueError:
            corr = 0
       
        corr = round(corr/60., 3)  # counts per minute to counts per sec
        press = round(press, 3)
        uncorr = round(ucorr/60., 3)  # counts per minute to counts per sec
        uncorr = check_data(uncorr, cr_min=10., cr_max=10000.)
        press = check_pressure(press, p_min=900., p_max=1100.)
        corr = check_data(corr, cr_min=10., cr_max=10000.)
        ceff = corr

        data.append([timestamp, interval, uncorr, ceff, corr, press])

    return data

## --------------------------------------------------------------------------
def __read_corrected(options, station, data_type, year=None, month=None, day=None):
    """OBSOLETE: use split_yearly and read_yearly instead
    read from big file with corrected data

    DD/MM/YYYY hh:mm[:ss] p_mbar ucorr corr,delta
    ~/Data/soft-derived/cosmic-rays/corrected-intensity/DOUR@NMDB/nm64-v1/1m/corrected_data_till_2012_10_09.dat 
    1/10/1965 00:00 983.485 6805.18 6651.19 -0.0388046
    1/10/1965 00:15 983.631 6815.44 6668.43 -0.0363137
...
31/12/2008 23:30        1002.25 6240.72 7006.72 0.0125734       
31/12/2008 23:45        1002.15 6199.34 6954.91 0.00508661      
2/01/2009       00:00:00        1000.12 6120    6763.74 -0.0225394
2/01/2009       00:01:00        1000.13 6213    6867.03 -0.00761256
...
9/10/2012       23:58:00        989.55  6177    6313.91 -0.0875475                      
9/10/2012       23:59:00        989.53  6290    6428.46 -0.0709927   
    """
    
    data = []    
    data_path = '~/data/dourbes'

    data_path = os.path.expanduser(data_path)
    if data_type in ["1m", "ori", "rev"]:
        data_type = "1m"
#        filename = '~/Data/soft-derived/cosmic-rays/corrected-intensity/DOUR@NMDB/nm64-v1/1m/corrected_data_till_2012_10_09.dat'
        filename = '~/data/dourbes/1m/corrected_data_till_2012_10_09.dat'
        filename = os.path.expanduser(filename)
        if year <= 2008:
            interval = 60*15
        else:
            interval = 60
    elif data_type in["1h"]:
        print "1h not defined yet"
        return data
    else:
        print "unknown data:", data_type
        return data

    try:
        myfile = open(filename, 'rb')
    except IOError:
        print "file not found", filename
        return data        

    line = True
    while line:
        line = myfile.readline()
        if (len(line) < 14) or (len(line) > 100):
            continue  # skip, line too short or too long for data
        row = line.split()

        date=row[0].split("/")
        if year and year != int(date[2]):
            continue  # skip data if year does not match
        if month and month != int(date[1]):
            continue  # skip data if month does not match
        if day and day != int(date[0]):
            continue  # skip data if day does not match

        time = row[1].split(":")
        hour = int(time[0])
        minute = int(time[1])

        timestamp = datetime(year, month, day, hour, minute)

        try:
            press = float(row[2])
        except ValueError:
            press = 0
        try:
            ucorr = float(row[3])
        except ValueError:
            ucorr = 0
        try:
            corr = float(row[4])
        except ValueError:
            corr = 0
       
        corr = round(corr/60., 3)  # counts per minute to counts per sec
        press = round(press, 3)
        uncorr = round(ucorr/60., 3)  # counts per minute to counts per sec
        uncorr = check_data(uncorr, cr_min=10., cr_max=10000.)
        press = check_pressure(press, p_min=900., p_max=1100.)
        corr = check_data(corr, cr_min=10., cr_max=10000.)
        ceff = corr

        data.append([timestamp, interval, uncorr, ceff, corr, press])

    return data

## --------------------------------------------------------------------------
def read_daily(options, station, data_type, year=None, month=None, day=None):
    """daily data files
    ~/Data/soft-derived/cosmic-rays/corrected-intensity/DOUR@NMDB/nm64-v1/1m/2015/01/Rm150101.corr 
    YYYY/MM/DD,hh:mm:ss,p_mbar,ucorr,corr,delta
    2012/10/09,00:00:00,983.92,6392,6267,-9.42572
    2012/10/09,00:01:00,983.9,6475,6347,-8.26317
    """
    
    if year is None or month is None or day is None:
        print "yeat, month, and day must be specified"
        raise ValueError

    data = []    
    data_path = '~/data/dourbes'

    data_path = os.path.expanduser(data_path)
    if data_type in ["1m", "ori", "rev"]:
        data_type = "1m"
        filename = "%s/1m/%04i/%02i/Rm%02i%02i%02i.corr" % (data_path, year, month, year2(year), month, day)
        if options.verbose:
            print filename
        interval = 60
    elif data_type in["1h"]:
        data = read_1h(options, station, year, month, day)
        return data
    else:
        print "unknown data:", data_type
        return data

    try:
        myfile = open(filename, 'rb')
    except IOError:
        print "file not found", filename
        return data        

    line = True
    while line:
        line = myfile.readline()
        if (len(line) < 14) or (len(line) > 100):
            continue  # skip, line too short or too long for data
        row = line.split(",")

        date=row[0].split("/")
        if year != int(date[0]):
            continue  # skip data if year does not match
        if month != int(date[1]):
            continue  # skip data if month does not match
        if day != int(date[2]):
            continue  # skip data if day does not match

        time = row[1].split(":")
        hour = int(time[0])
        minute = int(time[1])

        timestamp = datetime(year, month, day, hour, minute)

        try:
            press = float(row[2])
        except ValueError:
            press = 0
        try:
            ucorr = float(row[3])
        except ValueError:
            ucorr = 0
        try:
            corr = float(row[4])
        except ValueError:
            corr = 0

        corr = round(corr/interval, 3)
        press = round(press, 3)
        uncorr = round(ucorr/interval, 3)
        uncorr = check_data(uncorr, cr_min=10., cr_max=10000.)
        press = check_pressure(press, p_min=900., p_max=1100.)
        corr = check_data(corr, cr_min=10., cr_max=10000.)
        ceff = corr

        data.append([timestamp, interval, uncorr, ceff, corr, press])

    return data


## --------------------------------------------------------------------------
def main():

    import optparse
    parser = optparse.OptionParser("usage: %prog [options]")
    parser.add_option("-v", dest="verbose", default=0, action="count",
            help="increment output verbosity; may be specified multiple times")
    (options, args) = parser.parse_args()

    print "This is the dourbes module."

    station = "drbs"
    data_type="1h"
    #data_type="1m"
    year = 2013
#    year = 1965
    month = 12
    day = 13
    
    if True:
        #data = read_rev(options, station, data_type, year, month, day)
        data = read_rt(options, station, data_type, year, month, day)
        #data = read_corrected(options, station, data_type, year, month, day)
        #data = read_daily(options, station, data_type, year, month, day)
        #data = read_yearly(options, station, data_type, year, month, day)
        #data = read_1h(options, station, year, month, day)
        for line in data:
            print line
    else:
        # create yearly files:
        #data = split_corrected(options, station, data_type)
        for year in range(1965, 2015):
            data = read_yearly(options, station, data_type, year)
            for line in data:
                print line

## --------------------------------------------------------------------------
if __name__ == '__main__':
    main()
